package com.gxljc.bear.crawler.itaogao.cnstock;

import com.gxljc.bear.crawler.proxy.ProxyUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;

import java.io.Serializable;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

/**
 * 上海证券报 seed 数据爬虫。
 *
 * @author tanghaitao
 * @since 2022-9-20
 */
public class CnstockUpdateCrawler implements Serializable {
    public static Logger LOG = Logger.getLogger(CnstockUpdateCrawler.class);
    private Boolean proxy = false;
    private CnstockSeedCrawler crawler = null;

    public CnstockUpdateCrawler(Boolean proxy) {
        crawler = new CnstockSeedCrawler(proxy);
        this.proxy = proxy;
    }

    public void crawl() throws Exception {
        List<String> seeds = getSeed();
        if (CollectionUtils.isEmpty(seeds)) {
            LOG.error("page is empty");
            return;
        }
        ProxyUtil.initQueue(proxy);
        crawlNoSpark(seeds);
    }

    public void crawlNoSpark(List<String> seeds) {
        for (String url : seeds) {
            try {
                crawler.crawl(url);
            } catch (Exception e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            }
        }
    }

    //构造URL seed
    public List<String> getSeed() throws Exception {
        Map<String, Integer> seeds = new HashMap<String, Integer>();
        seeds.put("http://news.cnstock.com/news/sns_yw/%s", 1);
        seeds.put("http://company.cnstock.com/company/scp_gsxw/%s", 1);
        seeds.put("http://ggjd.cnstock.com/gglist/search/qmtbbdj/%s", 1);
        seeds.put("http://company.cnstock.com/company/scp_dsy/tcsy_rdgs/%s", 1);
        List<String> urls = new LinkedList<String>();
        for (Map.Entry seed : seeds.entrySet()) {
            int max = (Integer) seed.getValue();
            String urlFormat = (String) seed.getKey();
            for (int i = 1; i <= max; i++) {
                String url = String.format(urlFormat, i + "");
                urls.add(url);
            }
        }
        return urls;
    }

}
